package com.gxljc.bear.crawler.itaogao.huanqiu;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.ObjectUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.gxljc.commons.mongo.cli.BaseMongoRepository;
import com.gxljc.commons.util.StringUtil;
import com.gxljc.bear.crawler.base.DolphinFetchData;
import com.gxljc.bear.crawler.base.BaseMongoTools;
import com.gxljc.bear.crawler.base.DolphinCrawlerConsts;
import com.gxljc.bear.crawler.itaogao.ItaogaoConst;
import com.gxljc.bear.crawler.itaogao.NewsPageTable;
import com.gxljc.bear.crawler.itaogao.util.HbaseUtil;
import com.gxljc.bear.crawler.itaogao.util.HttpUtil;
import com.gxljc.bear.crawler.itaogao.util.ImageUtil;
import com.gxljc.bear.crawler.itaogao.util.MongodbUtil;
import com.gxljc.bear.crawler.util.DateUtil;

/**
 * @author haitao E-mail:haitao@summba.com
 * @version createTime：2021年9月27日 上午11:18:32
 * 
 */

public class HuanqiuCrawlerAlternative {

	private final static Logger LOG = Logger.getLogger(HuanqiuCrawlerAlternative.class);

	private final static List<String> crawlerSite = new ArrayList<String>();
	private final static String MONGONAME_COLLECTION = "t_site_huanqiu";
	private final static String MONGONAME_COLLECTION_SEED = "t_site_huanqiu_seed";
	private Map<String, Object> newsMap = new HashMap<String, Object>();
	private final static String SOURCE = "环球网";
	private static int countCrawler = 0;
	protected boolean proxy;
	private int pageNum = 1;

	public HuanqiuCrawlerAlternative() {
		crawlerSite.add("http://oversea.huanqiu.com/article/");// 海外看中国
		crawlerSite.add("http://world.huanqiu.com/article/");// 国际
		crawlerSite.add("http://opinion.huanqiu.com/roll.html");// 评论roll_2.html
		crawlerSite.add("http://mil.huanqiu.com/china/");// 中国军事
		crawlerSite.add("http://mil.huanqiu.com/world/");// 国际军事
		crawlerSite.add("http://mil.huanqiu.com/observation/");// 军事评论
		proxy = false;
		newsMap.put("status", 0);
		newsMap.put("mediaName", SOURCE);
	}

	/**
	 * =========爬虫启动入口(All)==========
	 */
	public void startCrawlerAll() {
		LOG.info("========Crawler seeds start!===========");
		genDefaultSeeds();
		LOG.info("Start crawler detail huanqiu news!");
		crawlerNews();
		LOG.info(String.format("huanqiu news crawler finished! and cralwer all size is %s", countCrawler));
	}

	// 获取种子列表
	void genDefaultSeeds() {
		for (String seed : crawlerSite) {
			// 判别网址是否存在
			if (!HttpUtil.checkUrlexists(seed)) {
				continue;
			}
			try {
				crawlerSeedsFirstPage(seed);
				LOG.info(String.format("page size is %s", pageNum));
			} catch (Exception e1) {
				LOG.info(String.format("firstPageNum crawler is error! and seed is %s", seed));
				e1.printStackTrace();
			}
			if (pageNum > 1) {
				for (int i = 2; i <= pageNum; i++) {
					String htmlNum = String.valueOf(i) + ".html";
					String url = "";
					if (seed.contains("roll.html")) {
						url = seed.replace("roll.html", "roll_" + htmlNum);
					} else {
						url = seed + htmlNum;
					}
					// 判别网址存在
					if (!HttpUtil.checkUrlexists(url)) {
						continue;
					}
					try {
						crawlPageSeed(url);
					} catch (Exception e) {
						LOG.info(String.format("crawler url is error!url is %s", url));
						e.printStackTrace();
					}
				}
			}

		}
	}

	/**
	 * 抓取新闻详情
	 */
	void crawlerNews() {
		BaseMongoRepository repo = BaseMongoTools.getInstance(DolphinCrawlerConsts.MongoDBName.ITAOGAO.getName());
		DBCollection rawDataColl = repo.getCollection(MONGONAME_COLLECTION_SEED);
		DBCursor cursor = rawDataColl.find();
		while (cursor.hasNext()) {
			DBObject currObj = cursor.next();
			String seed = ObjectUtils.toString(currObj.get("crawlerUrl"));
			LOG.info(String.format("crawlering url is %s", seed));
			String logoUrl = ObjectUtils.toString(currObj.get("logoUrl"));
			String title = ObjectUtils.toString(currObj.get("title"));
			String publishDate = ObjectUtils.toString(currObj.get("publishDate"));
			NewsPageTable table = new NewsPageTable();
			byte[] htmlByte = null;
			try {
				htmlByte = returnHtmlByte(seed);
			} catch (Exception e1) {
				LOG.info(String.format("crawler html Error! url is %s", htmlByte));
				e1.printStackTrace();
			}
			if (htmlByte != null) {
				String html = new String(htmlByte);

				// 基础数据抽取
				extractDetail(html, seed);

				String id = ObjectUtils.toString(StringUtil.genMD5Val(seed));
				String subTitle = "";

				if (StringUtils.isNotBlank(logoUrl)) {
					try {
						// 补齐相对图片路径
						logoUrl = ImageUtil.imageUrlSupply(seed, logoUrl);
						// 图片平台转换
						logoUrl = ImageUtil.imageConvertplatform(logoUrl);
					} catch (Exception e) {
						LOG.info(String.format("logoUrl convert failed! logoUrl is %s", logoUrl));
						e.printStackTrace();
					}
				}
				Date pd = DateUtil.parseFormat(publishDate, "yyyy-MM-dd HH:mm");
				if (pd != null) {
					newsMap.put("publishDate", pd);
				}
				// 添加基础数据
				newsMap.put("_id", id);
				newsMap.put("title", title);
				newsMap.put("subTitle", subTitle);
				newsMap.put("logoUrl", logoUrl);
				newsMap.put("crawlUrl", seed);
				newsMap.put("crawlFlag", 1);
				newsMap.put("crawlDate", new Date());

				// 更新mongdb
				MongodbUtil.update2Mongodb(newsMap, MONGONAME_COLLECTION);

				table.setTitle(title);
				table.setId(id);

				// 更新到hbase
				HbaseUtil.saveHbase(table, htmlByte);
				countCrawler += 1;
				LOG.info(String.format("insert the id/rowkey is %s ", id));
			}

		}
	}

	// 获取页码
	void crawlerSeedsFirstPage(String url) throws Exception {
		byte[] htmlbyte = returnHtmlByte(url);
		String num = extractPageNum(new String(htmlbyte));
		extract(new String(htmlbyte));
		if (StringUtils.isNotBlank(num)) {
			pageNum = Integer.parseInt(num) / 60;
			int re = Integer.parseInt(num) % 60;
			if (re > 0) {
				pageNum = pageNum + 1;
			}
		}
	}

	private String extractPageNum(String html) {
		String pageNum = "";
		Document doc = Jsoup.parse(html);
		Elements elements = doc.select("#pages a");
		for (Element element : elements.subList(0, 1)) {
			pageNum = element.text();
		}
		return pageNum.replace("条", "");
	}

	/**
	 * 下载网页并抽取入口
	 * 
	 * @param url
	 * @return
	 * @throws Exception
	 */
	public int crawlPageSeed(String url) throws Exception {
		DolphinFetchData fetchData = DolphinFetchData
				.getInstance(DolphinCrawlerConsts.CrawlerChannelType.bear.getName());
		byte[] htmlByte = fetchData.getHtml(url, proxy, ItaogaoConst.CRAWL_TIMEOUT);
		if (htmlByte == null) {
			LOG.error("page result is empty url = " + url);
			return DolphinCrawlerConsts.RESULT_NO;
		}
		String html = new String(htmlByte);
		if (StringUtils.isEmpty(html)) {
			LOG.error("page result is empty url = " + url);
			return DolphinCrawlerConsts.RESULT_NO;
		}
		extract(html);
		return DolphinCrawlerConsts.RESULT_YES;

	}

	// 抽取网页信息入口
	private int extract(String html) {
		Document doc = Jsoup.parse(html);
		Elements elements = doc.select(".fallsFlow ul li");
		if (CollectionUtils.isEmpty(elements))
			return 0;
		for (Element element : elements) {
			Map<String, Object> pageEntityInfo = extractPage(element);
			if (!pageEntityInfo.isEmpty()) {
				String crawlerUrl = ObjectUtils.toString(pageEntityInfo.get("crawlerUrl"));
				String id = StringUtil.genMD5Val(crawlerUrl);
				pageEntityInfo.put("_id", id);
				MongodbUtil.update2Mongodb(pageEntityInfo, MONGONAME_COLLECTION_SEED);
			}
		}
		return DolphinCrawlerConsts.RESULT_YES;
	}

	// 抽取html网页信息
	private Map<String, Object> extractPage(Element element) {
		Map<String, Object> re = new HashMap<String, Object>();
		Elements elements = element.select("a");
		Elements elements2 = element.select("h6");
		if (CollectionUtils.isEmpty(elements))
			return new HashMap<>();
		String title = elements.get(0).attr("title");
		String url = elements.get(0).attr("href");
		String logoUrl = extractLogoUrl(element);
		String publishDate = extractPublishDate(elements2);
		if (StringUtils.isEmpty(url))
			return new HashMap<>();
		// 若库内存在 continue
		if (!MongodbUtil.needReCrawl(url, MONGONAME_COLLECTION)) {
			re.put("title", title);
			re.put("crawlerUrl", url);
			re.put("logoUrl", logoUrl);
			re.put("publishDate", publishDate);
		}
		return re;
	}

	// 抽取logoUrl
	private String extractLogoUrl(Element element) {
		Elements elements = element.select("img");
		if (CollectionUtils.isEmpty(elements))
			return "";
		String image = elements.get(0).attr("src");
		return image;
	}

	// 抽取日期
	private String extractPublishDate(Elements elements) {
		return elements.get(0).text();
	}

	void crawlerNewsSingle(Map<String, Object> map) {
		String seed = ObjectUtils.toString(map.get("crawlerUrl"));
		LOG.info(String.format("crawlering url is %s", seed));
		String logoUrl = ObjectUtils.toString(map.get("logoUrl"));
		String title = ObjectUtils.toString(map.get("title"));
		String publishDate = ObjectUtils.toString(map.get("publishDate"));
		NewsPageTable table = new NewsPageTable();
		byte[] htmlByte = null;
		try {
			htmlByte = returnHtmlByte(seed);
		} catch (Exception e1) {
			LOG.info(String.format("crawler html Error! url is %s", htmlByte));
			e1.printStackTrace();
		}
		if (htmlByte != null) {
			String html = new String(htmlByte);

			// 基础数据抽取
			extractDetail(html, seed);

			String id = ObjectUtils.toString(StringUtil.genMD5Val(seed));
			String subTitle = "";

			if (StringUtils.isNotBlank(logoUrl)) {
				try {
					// 补齐相对图片路径
					logoUrl = ImageUtil.imageUrlSupply(seed, logoUrl);
					// 图片平台转换
					logoUrl = ImageUtil.imageConvertplatform(logoUrl);
				} catch (Exception e) {
					LOG.info(String.format("logoUrl convert failed! logoUrl is %s", logoUrl));
					e.printStackTrace();
				}
			}
			Date pd = DateUtil.parseFormat(publishDate, "yyyy-MM-dd HH:mm");
			if (pd != null) {
				newsMap.put("publishDate", pd);
			}
			// 添加基础数据
			newsMap.put("_id", id);
			newsMap.put("title", title);
			newsMap.put("subTitle", subTitle);
			newsMap.put("logoUrl", logoUrl);
			newsMap.put("crawlUrl", seed);
			newsMap.put("crawlFlag", 1);
			newsMap.put("crawlDate", new Date());

			// 更新mongdb
			MongodbUtil.update2Mongodb(newsMap, MONGONAME_COLLECTION);

			table.setTitle(title);
			table.setId(id);

			// 更新到hbase
			HbaseUtil.saveHbase(table, htmlByte);
			countCrawler += 1;
			LOG.info(String.format("insert the id/rowkey is %s ", id));
		}
	}

	/*
	 * private String extractContent(String html) { List<String> contentList =
	 * new ArrayList<String>(); Document doc = Jsoup.parse(html); Elements
	 * elements = doc.select("#text p"); if (CollectionUtils.isEmpty(elements))
	 * return ""; for (Element element : elements) {
	 * if(element.html().contains("href") || element.html().contains("strong")){
	 * String text = element.html().replaceAll("<(.[^>]*)>", "");
	 * contentList.add(text); continue; } contentList.add(element.text()); }
	 * 
	 * return StringUtils.join(contentList, "####"); }
	 */

	private String extractSource(Document doc) {
		String source = "";
		Elements element = doc.select("#source_baidu a");
		source = element.text();
		return source;
	}

	private String extractImages(Element element) {
		Elements elements = element.select("img");
		if (CollectionUtils.isEmpty(elements))
			return "";
		String image = elements.get(0).attr("src");
		return image;
	}

	private List<String> extractAuthorPop(Document doc) {
		List<String> authorPop = new ArrayList<String>();
		Elements elements = doc.select("#authorPop ul li");
		for (Element element : elements) {
			authorPop.add(element.text());
		}
		return authorPop;
	}

	private String extractAuthor(Document doc) {
		String author = "";
		Elements element = doc.select("#author_baidu");
		author = element.text();

		return author;
	}

	private byte[] returnHtmlByte(String seed) throws Exception {
		DolphinFetchData fetchData = DolphinFetchData
				.getInstance(DolphinCrawlerConsts.CrawlerChannelType.bear.getName());
		byte[] htmlByte = fetchData.getHtml(seed, proxy, ItaogaoConst.CRAWL_TIMEOUT);
		if (htmlByte == null) {
			LOG.error("page result is empty url = " + seed);
			return null;
		}
		return htmlByte;
	}

	/**
	 * 插入content,source,images,author
	 * 
	 * @param html
	 * @param url
	 */
	private void extractDetail(String html, String url) {
		List<String> images = new ArrayList<String>();
		List<String> authors = new ArrayList<String>();
		List<String> contents = new ArrayList<String>();
		List<String> imagesNew = new ArrayList<String>();
		Document doc = Jsoup.parse(html);
		Elements elements = doc.select("#text p");
		Elements elementsImages = doc.select("#picWrap img");
		String source = extractSource(doc);

		for (Element element : elementsImages) {
			images.add(element.attr("src"));
		}

		authors.add(extractAuthor(doc));
		authors.addAll(extractAuthorPop(doc));

		if (!CollectionUtils.isEmpty(elements)) {
			for (Element element : elements) {
				if (element.html().contains("href") || element.html().contains("strong")) {
					String text = element.html().replaceAll("<(.[^>]*)>", "");
					contents.add(text);
					continue;
				}
				String imageInner = extractImages(element);
				if (!imageInner.isEmpty())
					images.add(imageInner);
				contents.add(element.text());

			}

		}
		putListToMap(authors, "author", true, ",");
		putListToMap(contents, "content", true, "####");
		newsMap.put("source", source);
		if (images.size() > 0) {
			imagesNew = ImageUtil.ImageUrlService(images, url);
		}
		if (imagesNew.size() != 0) {
			newsMap.put("images", imagesNew);
		}
	}

	@SuppressWarnings("unchecked")
	void putListToMap(Object listObject, String putKey, boolean joinFlag, String splitMark) {
		List<String> list = new ArrayList<String>();
		if (listObject != null) {
			list = (List<String>) listObject;
		}
		// 是否需要拼接
		if (joinFlag) {
			if (CollectionUtils.isNotEmpty(list)) {
				newsMap.put(putKey, StringUtils.join(list, splitMark).replace("####　　", "####")
						.replace("AD_SURVEY_Add_AdPos(\"14213\");", ""));
			}
		} else {
			if (CollectionUtils.isNotEmpty(list)) {
				newsMap.put(putKey, list);
			}

		}
	}

}
